{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": true,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "from keras.layers import Dense, Input, Flatten, concatenate, Conv1D\n",
    "from keras.layers import GlobalAveragePooling1D, GlobalMaxPooling1D,Bidirectional, Convolution1D, Embedding, SpatialDropout1D, BatchNormalization,MaxPooling1D, Dropout, LSTM, GRU\n",
    "from keras import backend as K\n",
    "from keras.engine.topology import Layer\n",
    "from keras import initializers, regularizers, constraints\n",
    "from keras.models import Model\n",
    "from keras.layers.merge import Concatenate\n",
    "from keras.callbacks import EarlyStopping, ModelCheckpoint"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "INPUT_PATH = '../input/'\n",
    "CACHE_PATH = '../cache/'\n",
    "OUTPUT_PATH ='../output/'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "data = np.load(CACHE_PATH + 'data.npz')\n",
    "X_train = data['X_train']\n",
    "y_train = data['y_train']\n",
    "X_val = data['X_val']\n",
    "y_val = data['y_val']\n",
    "X_test = data['X_test']\n",
    "embedding_matrix = np.load(CACHE_PATH + 'embedding_matrix.npy')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "MAX_FEATURES = 20000\n",
    "MAX_SEQUENCE_LENGTH = 100\n",
    "EMBEDDING_DIM = 128"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "embedding_layer = Embedding(MAX_FEATURES,\n",
    "                            EMBEDDING_DIM,\n",
    "                            weights=[embedding_matrix],\n",
    "                            input_length=MAX_SEQUENCE_LENGTH,\n",
    "                            trainable=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def get_gru_model():\n",
    "    inp = Input(shape=(MAX_SEQUENCE_LENGTH,), dtype='int32')\n",
    "    embedded_sequences = embedding_layer(inp)\n",
    "    x = SpatialDropout1D(0.2)(embedded_sequences)\n",
    "    x = Bidirectional(GRU(250, dropout=0.25, recurrent_dropout=0.25, return_sequences=True))(embedded_sequences)\n",
    "    x = Conv1D(64, kernel_size = 3, padding = \"valid\", kernel_initializer = \"glorot_uniform\")(x)\n",
    "    avg_pool = GlobalAveragePooling1D()(x)\n",
    "    max_pool = GlobalMaxPooling1D()(x)\n",
    "    x = concatenate([avg_pool, max_pool])\n",
    "    x = Dense(100, activation=\"relu\")(x)\n",
    "    x = Dropout(0.1)(x)\n",
    "    x = Dense(1, activation=\"linear\")(x)\n",
    "    model = Model(inputs=inp, outputs=x)\n",
    "    model.compile(loss='mse',optimizer='adam')\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def train_gru_model(model):\n",
    "    model_path = CACHE_PATH + \"gru_weights_best.hdf5\"\n",
    "    early = EarlyStopping(monitor=\"val_loss\", mode=\"min\", patience=5)\n",
    "    checkpoint = ModelCheckpoint(model_path, monitor='val_loss', verbose=1, save_best_only=True, mode='min')\n",
    "    callbacks_list = [checkpoint, early]\n",
    "    model.fit(X_train, y_train, batch_size=128, epochs=100, validation_data=(X_val, y_val), callbacks=callbacks_list)\n",
    "    model.load_weights(model_path)\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 209000 samples, validate on 11000 samples\n",
      "Epoch 1/100\n",
      "208896/209000 [============================>.] - ETA: 0s - loss: 0.6088\n",
      "Epoch 00001: val_loss improved from inf to 0.45478, saving model to ../cache/gru_weights_best.hdf5\n",
      "209000/209000 [==============================] - 380s 2ms/step - loss: 0.6088 - val_loss: 0.4548\n",
      "Epoch 2/100\n",
      "208896/209000 [============================>.] - ETA: 0s - loss: 0.5109\n",
      "Epoch 00002: val_loss improved from 0.45478 to 0.43517, saving model to ../cache/gru_weights_best.hdf5\n",
      "209000/209000 [==============================] - 379s 2ms/step - loss: 0.5110 - val_loss: 0.4352\n",
      "Epoch 3/100\n",
      "208896/209000 [============================>.] - ETA: 0s - loss: 0.4884\n",
      "Epoch 00003: val_loss did not improve\n",
      "209000/209000 [==============================] - 377s 2ms/step - loss: 0.4885 - val_loss: 0.4536\n",
      "Epoch 4/100\n",
      "208896/209000 [============================>.] - ETA: 0s - loss: 0.4686\n",
      "Epoch 00004: val_loss did not improve\n",
      "209000/209000 [==============================] - 379s 2ms/step - loss: 0.4686 - val_loss: 0.4381\n",
      "Epoch 5/100\n",
      "208896/209000 [============================>.] - ETA: 0s - loss: 0.4561\n",
      "Epoch 00005: val_loss improved from 0.43517 to 0.41985, saving model to ../cache/gru_weights_best.hdf5\n",
      "209000/209000 [==============================] - 377s 2ms/step - loss: 0.4561 - val_loss: 0.4199\n",
      "Epoch 6/100\n",
      "208896/209000 [============================>.] - ETA: 0s - loss: 0.4451\n",
      "Epoch 00006: val_loss did not improve\n",
      "209000/209000 [==============================] - 379s 2ms/step - loss: 0.4451 - val_loss: 0.4564\n",
      "Epoch 7/100\n",
      "208896/209000 [============================>.] - ETA: 0s - loss: 0.4340\n",
      "Epoch 00007: val_loss improved from 0.41985 to 0.41398, saving model to ../cache/gru_weights_best.hdf5\n",
      "209000/209000 [==============================] - 376s 2ms/step - loss: 0.4340 - val_loss: 0.4140\n",
      "Epoch 8/100\n",
      "208896/209000 [============================>.] - ETA: 0s - loss: 0.4260\n",
      "Epoch 00008: val_loss did not improve\n",
      "209000/209000 [==============================] - 371s 2ms/step - loss: 0.4259 - val_loss: 0.4704\n",
      "Epoch 9/100\n",
      "208896/209000 [============================>.] - ETA: 0s - loss: 0.4182\n",
      "Epoch 00009: val_loss did not improve\n",
      "209000/209000 [==============================] - 368s 2ms/step - loss: 0.4181 - val_loss: 0.4184\n",
      "Epoch 10/100\n",
      "208896/209000 [============================>.] - ETA: 0s - loss: 0.4092\n",
      "Epoch 00010: val_loss did not improve\n",
      "209000/209000 [==============================] - 368s 2ms/step - loss: 0.4092 - val_loss: 0.4208\n",
      "Epoch 11/100\n",
      "208896/209000 [============================>.] - ETA: 0s - loss: 0.4050\n",
      "Epoch 00011: val_loss did not improve\n",
      "209000/209000 [==============================] - 370s 2ms/step - loss: 0.4050 - val_loss: 0.4158\n",
      "Epoch 12/100\n",
      "208896/209000 [============================>.] - ETA: 0s - loss: 0.3987\n",
      "Epoch 00012: val_loss improved from 0.41398 to 0.41291, saving model to ../cache/gru_weights_best.hdf5\n",
      "209000/209000 [==============================] - 371s 2ms/step - loss: 0.3988 - val_loss: 0.4129\n",
      "Epoch 13/100\n",
      "208896/209000 [============================>.] - ETA: 0s - loss: 0.3945\n",
      "Epoch 00013: val_loss did not improve\n",
      "209000/209000 [==============================] - 371s 2ms/step - loss: 0.3944 - val_loss: 0.4241\n",
      "Epoch 14/100\n",
      "208896/209000 [============================>.] - ETA: 0s - loss: 0.3917\n",
      "Epoch 00014: val_loss improved from 0.41291 to 0.40838, saving model to ../cache/gru_weights_best.hdf5\n",
      "209000/209000 [==============================] - 371s 2ms/step - loss: 0.3917 - val_loss: 0.4084\n",
      "Epoch 15/100\n",
      "208896/209000 [============================>.] - ETA: 0s - loss: 0.3871\n",
      "Epoch 00015: val_loss did not improve\n",
      "209000/209000 [==============================] - 371s 2ms/step - loss: 0.3871 - val_loss: 0.4496\n",
      "Epoch 16/100\n",
      "208896/209000 [============================>.] - ETA: 0s - loss: 0.3854\n",
      "Epoch 00016: val_loss did not improve\n",
      "209000/209000 [==============================] - 371s 2ms/step - loss: 0.3854 - val_loss: 0.4175\n",
      "Epoch 17/100\n",
      "208896/209000 [============================>.] - ETA: 0s - loss: 0.3826\n",
      "Epoch 00017: val_loss did not improve\n",
      "209000/209000 [==============================] - 371s 2ms/step - loss: 0.3826 - val_loss: 0.4238\n",
      "Epoch 18/100\n",
      "208896/209000 [============================>.] - ETA: 0s - loss: 0.3795\n",
      "Epoch 00018: val_loss did not improve\n",
      "209000/209000 [==============================] - 370s 2ms/step - loss: 0.3795 - val_loss: 0.4089\n",
      "Epoch 19/100\n",
      "208896/209000 [============================>.] - ETA: 0s - loss: 0.3775\n",
      "Epoch 00019: val_loss improved from 0.40838 to 0.40640, saving model to ../cache/gru_weights_best.hdf5\n",
      "209000/209000 [==============================] - 371s 2ms/step - loss: 0.3775 - val_loss: 0.4064\n",
      "Epoch 20/100\n",
      "208896/209000 [============================>.] - ETA: 0s - loss: 0.3786\n",
      "Epoch 00020: val_loss did not improve\n",
      "209000/209000 [==============================] - 369s 2ms/step - loss: 0.3785 - val_loss: 0.4126\n",
      "Epoch 21/100\n",
      "208896/209000 [============================>.] - ETA: 0s - loss: 0.3743\n",
      "Epoch 00021: val_loss did not improve\n",
      "209000/209000 [==============================] - 371s 2ms/step - loss: 0.3742 - val_loss: 0.4081\n",
      "Epoch 22/100\n",
      "208896/209000 [============================>.] - ETA: 0s - loss: 0.3716\n",
      "Epoch 00022: val_loss did not improve\n",
      "209000/209000 [==============================] - 371s 2ms/step - loss: 0.3716 - val_loss: 0.4210\n",
      "Epoch 23/100\n",
      "208896/209000 [============================>.] - ETA: 0s - loss: 0.3725\n",
      "Epoch 00023: val_loss did not improve\n",
      "209000/209000 [==============================] - 371s 2ms/step - loss: 0.3724 - val_loss: 0.4406\n",
      "Epoch 24/100\n",
      "208896/209000 [============================>.] - ETA: 0s - loss: 0.3727\n",
      "Epoch 00024: val_loss did not improve\n",
      "209000/209000 [==============================] - 370s 2ms/step - loss: 0.3727 - val_loss: 0.4091\n"
     ]
    }
   ],
   "source": [
    "model = get_gru_model()\n",
    "model = train_gru_model(model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "50000/50000 [==============================] - 22s 433us/step\n"
     ]
    }
   ],
   "source": [
    "y_test = model.predict(X_test,batch_size=128,verbose=1)\n",
    "y_test[y_test < 1] = 1\n",
    "y_test[y_test > 5] = 5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "sub = pd.read_csv(INPUT_PATH + 'sample.csv',header=None,names=['Id','Score'])\n",
    "sub['Score'] = y_test\n",
    "sub.to_csv(OUTPUT_PATH + 'gru.csv',index=False, header=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
